{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualization for Ground Truth* and Predicted Trajectories\n",
    "\n",
    "\\* Shortest Path\n",
    "\n",
    "### Instructions:\n",
    "1. modified `3D/CheckConnectivity`\n",
    "2. start a local server under directory `3D` by \n",
    "``` bash\n",
    "# bash\n",
    "cd preprocess/3D\n",
    "python3 -m http.server```\n",
    "3. make sure following paths are correct: \n",
    "``` python\n",
    "# ground truth trajectory\n",
    "trajectory_path = \"tasks/R2R/results/val_seen_shortest_agent.json\"  \n",
    "# prediction trajectory\n",
    "trajectory_path = \"tasks/R2R/Nresults/seq2seq_sample_imagenet_val_seen_iter_16500.json\"  \n",
    "# instructions\n",
    "instruction_path = \"tasks/R2R/data/R2R_val_seen.json\"\n",
    "# connectivity information (where %s will be a scan ID)\n",
    "graph_path = \"connectivity/%s_connectivity.json\"  \n",
    "```\n",
    "4. Never run the whole notebook together. You have to wait for `imgData= browser.execute_script...` to load images\n",
    "5. After running the notebook, `./jolin_mesh_names.json` and `'./%s.json'% scan` will be genereated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json, os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get all folders in mesh\n",
    "folders = {}\n",
    "files = os.listdir(\"./matterport_mesh/v1/scans/\")\n",
    "for id, name in enumerate(files):\n",
    "    subfolder = os.listdir(\"./matterport_mesh/v1/scans/\" + name + \"/matterport_mesh/\")\n",
    "    folders[name]=subfolder[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def idx2scan_folder(idx, trajectory_data):\n",
    "    trajectory = trajectory_data[idx]\n",
    "    instr_id = trajectory['instr_id']\n",
    "    scan = instr_id2scan[instr_id]\n",
    "    txt = instr_id2txt[instr_id]\n",
    "    return [scan, folders[scan]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def traj2conn_json(graph_path, idx, trajectory_data):\n",
    "    trajectory = trajectory_data[idx]\n",
    "    instr_id = trajectory['instr_id']\n",
    "    scan = instr_id2scan[instr_id]\n",
    "    viewpointId2idx={}\n",
    "    with open(graph_path % scan) as f:\n",
    "        conn_data = json.load(f)\n",
    "    for i,item in enumerate(conn_data):\n",
    "        viewpointId2idx[item['image_id']]=i\n",
    "    return trajectory, viewpointId2idx, conn_data\n",
    "\n",
    "def gen_conns(trajectory, viewpointId2idx, conn_data):\n",
    "    trajectory = trajectory_data[idx]\n",
    "    node=conn_data[viewpointId2idx[trajectory['trajectory'][0][0]]]\n",
    "    node={k:v for k,v in node.items()}\n",
    "    node['unobstructed'] = [False]*len(trajectory['trajectory'])\n",
    "    conns=[node]\n",
    "    prev_viewpoint = node['image_id']\n",
    "    for n, (viewpoint, heading, elevation) in enumerate(trajectory['trajectory'][1:]):\n",
    "        node=conn_data[viewpointId2idx[viewpoint]]\n",
    "        node={k:v for k,v in node.items()}\n",
    "        prev_viewpoint = conns[-1]['image_id']\n",
    "        if viewpoint != prev_viewpoint:\n",
    "            assert node['unobstructed'][viewpointId2idx[prev_viewpoint]]\n",
    "            node['unobstructed'] = [False]*len(trajectory['trajectory'])\n",
    "            node['unobstructed'][len(conns)-1]=True\n",
    "            conns.append(node)\n",
    "    return conns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_dicts(trajectory_path, instruction_path):\n",
    "    with open(trajectory_path) as f:\n",
    "        trajectory_data = json.load(f)\n",
    "    with open(instruction_path) as f:\n",
    "        instruction_data = json.load(f)\n",
    "\n",
    "    instr_id2txt = {\n",
    "        ('%s_%d' % (d['path_id'], n)): txt for d in instruction_data for n, txt in enumerate(d['instructions'])}\n",
    "    instr_id2scan = {\n",
    "        ('%s_%d' % (d['path_id'], n)): d['scan'] for d in instruction_data for n, txt in enumerate(d['instructions'])}\n",
    "    scan2trajidx = {\n",
    "        instr_id2scan[traj['instr_id']]:idx for idx, traj in enumerate(trajectory_data)}\n",
    "    instr_id2trajidx = {\n",
    "        traj['instr_id']:idx for idx, traj in enumerate(trajectory_data)}\n",
    "    return trajectory_data, instruction_data, instr_id2txt, instr_id2scan, scan2trajidx, instr_id2trajidx"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Start a local server"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`cd /home/xql/Source/Subgoal/preprocess/3D\n",
    "python3 -m http.server 8001`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "from IPython.display import display, HTML\n",
    "\n",
    "options = Options()\n",
    "options.headless = True\n",
    "browser = webdriver.Chrome(executable_path=\"/home/nav/Tools/chromedriver\", options=options)\n",
    "url = \"http://172.28.3.57:8001/connectivity.html\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Specify the trajectory ID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "instr_id =\"6077_2\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ground Truth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[u'zsNo4HB9uLZ', 'a641c3f4647545a2a4f5c50f5f5fbb57']]\n",
      "6077_2\n",
      "Exit the sitting room and turn left. Take the first turn to the right.Go straight and then turn left at the shelf. Walk into the first door on the right. \n",
      "\n",
      "Turn around and go passed the couch, then turn left and take an immediate right passed the painting. Turn left and go straight through the first door on the left. \n",
      "Turn left and exit the room.  Cross the hall to the sitting room.  Turn left and enter the bedroom on your left.  Wait near the bed. \n",
      "Exit the sitting room and turn left. Take the first turn to the right.Go straight and then turn left at the shelf. Walk into the first door on the right. \n"
     ]
    }
   ],
   "source": [
    "trajectory_path = \"/home/xql/Source/Subgoal/tasks/R2R/results/val_unseen_shortest_agent.json\"\n",
    "\n",
    "instruction_path = \"/home/xql/Source/Subgoal/tasks/R2R/data/R2R_val_unseen.json\"\n",
    "\n",
    "graph_path = \"/home/xql/Source/Subgoal/connectivity/%s_connectivity.json\"\n",
    "\n",
    "trajectory_data, instruction_data, instr_id2txt, instr_id2scan, scan2trajidx, instr_id2trajidx \\\n",
    "= build_dicts(trajectory_path, instruction_path)\n",
    "\n",
    "idxs= [instr_id2trajidx[instr_id]]\n",
    "scan_folders = [idx2scan_folder(idx, trajectory_data) for idx in idxs]\n",
    "print(scan_folders)\n",
    "instr_id = trajectory_data[idxs[0]]['instr_id']\n",
    "print(instr_id)\n",
    "\n",
    "# show instructions\n",
    "\n",
    "instruction = instr_id2txt[instr_id]\n",
    "print(instruction)\n",
    "print('')\n",
    "for i in ['0','1','2']:\n",
    "    print(instr_id2txt[instr_id[:-1]+i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dump connections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "scan_folders = [idx2scan_folder(idx, trajectory_data) for idx in idxs]\n",
    "\n",
    "with open('./jolin_mesh_names.json', 'w') as fp:\n",
    "    json.dump(scan_folders, fp)\n",
    "\n",
    "for idx, (scan, folder) in zip(idxs, scan_folders):\n",
    "    with open('./%s.json'% scan, 'w') as fp:\n",
    "        trajectory, viewpointId2idx, conn_data=traj2conn_json(graph_path, idx, trajectory_data)\n",
    "        json.dump(gen_conns(trajectory, viewpointId2idx, conn_data), fp)\n",
    "\n",
    "browser.get(url) #navigate to the page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<img src=\"\">"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "imgData= browser.execute_script('return renderer.domElement.toDataURL().replace(\"image/png\", \"image/octet-stream\")')\n",
    "display(HTML('''<img src=\"%s\">'''%(imgData)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exit the sitting room and turn left. Take the first turn to the right.Go straight and then turn left at the shelf. Walk into the first door on the right. \n"
     ]
    }
   ],
   "source": [
    "# Predication1\n",
    "trajectory_path = \"/home/xql/Source/Subgoal/tasks/R2R/exps/Nresult-len8-bi-mean/seq2seq_sample_imagenet_val_unseen_iter_36000.json\"\n",
    "\n",
    "trajectory_data, instruction_data, instr_id2txt, instr_id2scan, scan2trajidx, instr_id2trajidx \\\n",
    "= build_dicts(trajectory_path, instruction_path)\n",
    "\n",
    "idxs= [instr_id2trajidx[instr_id]]\n",
    "print(instr_id2txt[instr_id])\n",
    "\n",
    "### Dump connections\n",
    "\n",
    "scan_folders = [idx2scan_folder(idx, trajectory_data) for idx in idxs]\n",
    "\n",
    "with open('./jolin_mesh_names.json', 'w') as fp:\n",
    "    json.dump(scan_folders, fp)\n",
    "\n",
    "for idx, (scan, folder) in zip(idxs, scan_folders):\n",
    "    with open('./%s.json'% scan, 'w') as fp:\n",
    "        trajectory, viewpointId2idx, conn_data=traj2conn_json(graph_path, idx, trajectory_data)\n",
    "        json.dump(gen_conns(trajectory, viewpointId2idx, conn_data), fp)\n",
    "\n",
    "browser.get(url) #navigate to the page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<img src=\"\">"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "imgData= browser.execute_script('return renderer.domElement.toDataURL().replace(\"image/png\", \"image/octet-stream\")')\n",
    "display(HTML('''<img src=\"%s\">'''%(imgData)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exit the sitting room and turn left. Take the first turn to the right.Go straight and then turn left at the shelf. Walk into the first door on the right. \n"
     ]
    }
   ],
   "source": [
    "# Predication2\n",
    "\n",
    "trajectory_path = \"/home/nav/Source/speaker_follower/tasks/R2R/eval_outputs/pragmatics_val_unseen_speaker_weight_0.00.json\"\n",
    "\n",
    "trajectory_data, instruction_data, instr_id2txt, instr_id2scan, scan2trajidx, instr_id2trajidx \\\n",
    "= build_dicts(trajectory_path, instruction_path)\n",
    "\n",
    "idxs= [instr_id2trajidx[instr_id]]\n",
    "print(instr_id2txt[instr_id])\n",
    "\n",
    "### Dump connections\n",
    "\n",
    "scan_folders = [idx2scan_folder(idx, trajectory_data) for idx in idxs]\n",
    "\n",
    "with open('./jolin_mesh_names.json', 'w') as fp:\n",
    "    json.dump(scan_folders, fp)\n",
    "\n",
    "for idx, (scan, folder) in zip(idxs, scan_folders):\n",
    "    with open('./%s.json'% scan, 'w') as fp:\n",
    "        trajectory, viewpointId2idx, conn_data=traj2conn_json(graph_path, idx, trajectory_data)\n",
    "        json.dump(gen_conns(trajectory, viewpointId2idx, conn_data), fp)\n",
    "\n",
    "browser.get(url) #navigate to the page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<img src=\"\">"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "imgData= browser.execute_script('return renderer.domElement.toDataURL().replace(\"image/png\", \"image/octet-stream\")')\n",
    "display(HTML('''<img src=\"%s\">'''%(imgData)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exit the sitting room and turn left. Take the first turn to the right.Go straight and then turn left at the shelf. Walk into the first door on the right. \n"
     ]
    }
   ],
   "source": [
    "# Predication3\n",
    "\n",
    "trajectory_path = \"/home/xql/Source/Subgoal/tasks/R2R/exps/baseline/seq2seq_sample_imagenet_val_unseen_iter_13100.json\"\n",
    "\n",
    "trajectory_data, instruction_data, instr_id2txt, instr_id2scan, scan2trajidx, instr_id2trajidx \\\n",
    "= build_dicts(trajectory_path, instruction_path)\n",
    "\n",
    "idxs= [instr_id2trajidx[instr_id]]\n",
    "print(instr_id2txt[instr_id])\n",
    "\n",
    "### Dump connections\n",
    "\n",
    "scan_folders = [idx2scan_folder(idx, trajectory_data) for idx in idxs]\n",
    "\n",
    "with open('./jolin_mesh_names.json', 'w') as fp:\n",
    "    json.dump(scan_folders, fp)\n",
    "\n",
    "for idx, (scan, folder) in zip(idxs, scan_folders):\n",
    "    with open('./%s.json'% scan, 'w') as fp:\n",
    "        trajectory, viewpointId2idx, conn_data=traj2conn_json(graph_path, idx, trajectory_data)\n",
    "        json.dump(gen_conns(trajectory, viewpointId2idx, conn_data), fp)\n",
    "\n",
    "browser.get(url) #navigate to the page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<img src=\"\">"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "imgData= browser.execute_script('return renderer.domElement.toDataURL().replace(\"image/png\", \"image/octet-stream\")')\n",
    "display(HTML('''<img src=\"%s\">'''%(imgData)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py2caffe",
   "language": "python",
   "name": "py2caffe"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
